nnetsauce

 19class AdaBoostClassifier(Boosting, ClassifierMixin):
 20    """AdaBoost Classification (SAMME) model class derived from class Boosting
 21
 22    Parameters:
 23
 24        obj: object
 25            any object containing a method fit (obj.fit()) and a method predict
 26            (obj.predict())
 27
 28        n_estimators: int
 29            number of boosting iterations
 30
 31        learning_rate: float
 32            learning rate of the boosting procedure
 33
 34        n_hidden_features: int
 35            number of nodes in the hidden layer
 36
 37        reg_lambda: float
 38            regularization parameter for weights
 39
 40        reg_alpha: float
 41            controls compromize between l1 and l2 norm of weights
 42
 43        activation_name: str
 44            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 45
 46        a: float
 47            hyperparameter for 'prelu' or 'elu' activation function
 48
 49        nodes_sim: str
 50            type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
 51            'uniform'
 52
 53        bias: boolean
 54            indicates if the hidden layer contains a bias term (True) or not
 55            (False)
 56
 57        dropout: float
 58            regularization parameter; (random) percentage of nodes dropped out
 59            of the training
 60
 61        direct_link: boolean
 62            indicates if the original predictors are included (True) in model's
 63            fitting or not (False)
 64
 65        n_clusters: int
 66            number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
 67                no clustering)
 68
 69        cluster_encode: bool
 70            defines how the variable containing clusters is treated (default is one-hot)
 71            if `False`, then labels are used, without one-hot encoding
 72
 73        type_clust: str
 74            type of clustering method: currently k-means ('kmeans') or Gaussian
 75            Mixture Model ('gmm')
 76
 77        type_scaling: a tuple of 3 strings
 78            scaling methods for inputs, hidden layer, and clustering respectively
 79            (and when relevant).
 80            Currently available: standardization ('std') or MinMax scaling ('minmax')
 81
 82        col_sample: float
 83            percentage of covariates randomly chosen for training
 84
 85        row_sample: float
 86            percentage of rows chosen for training, by stratified bootstrapping
 87
 88        seed: int
 89            reproducibility seed for nodes_sim=='uniform'
 90
 91        method: str
 92            type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
 93
 94        backend: str
 95            "cpu" or "gpu" or "tpu"
 96
 97    Attributes:
 98
 99        alpha_: list
100            AdaBoost coefficients alpha_m
101
102        base_learners_: dict
103            a dictionary containing the base learners
104
105    Examples:
106
107    See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py)
108
109    ```python
110    import nnetsauce as ns
111    import numpy as np
112    from sklearn.datasets import load_breast_cancer
113    from sklearn.linear_model import LogisticRegression
114    from sklearn.model_selection import train_test_split
115    from sklearn import metrics
116    from time import time
117
118    breast_cancer = load_breast_cancer()
119    Z = breast_cancer.data
120    t = breast_cancer.target
121    np.random.seed(123)
122    X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
123
124    # SAMME.R
125    clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
126                            random_state=123)
127    fit_obj = ns.AdaBoostClassifier(clf,
128                                    n_hidden_features=int(11.22338867),
129                                    direct_link=True,
130                                    n_estimators=250, learning_rate=0.01126343,
131                                    col_sample=0.72684326, row_sample=0.86429443,
132                                    dropout=0.63078613, n_clusters=2,
133                                    type_clust="gmm",
134                                    verbose=1, seed = 123,
135                                    method="SAMME.R")
136
137    start = time()
138    fit_obj.fit(X_train, y_train)
139    print(f"Elapsed {time() - start}")
140
141    start = time()
142    print(fit_obj.score(X_test, y_test))
143    print(f"Elapsed {time() - start}")
144
145    preds = fit_obj.predict(X_test)
146
147    print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
148    print(metrics.classification_report(preds, y_test))
149
150    ```
151
152    """
153
154    # construct the object -----
155
156    def __init__(
157        self,
158        obj,
159        n_estimators=10,
160        learning_rate=0.1,
161        n_hidden_features=1,
162        reg_lambda=0,
163        reg_alpha=0.5,
164        activation_name="relu",
165        a=0.01,
166        nodes_sim="sobol",
167        bias=True,
168        dropout=0,
169        direct_link=False,
170        n_clusters=2,
171        cluster_encode=True,
172        type_clust="kmeans",
173        type_scaling=("std", "std", "std"),
174        col_sample=1,
175        row_sample=1,
176        seed=123,
177        verbose=1,
178        method="SAMME",
179        backend="cpu",
180    ):
181        super().__init__(
182            obj=obj,
183            n_estimators=n_estimators,
184            learning_rate=learning_rate,
185            n_hidden_features=n_hidden_features,
186            activation_name=activation_name,
187            a=a,
188            nodes_sim=nodes_sim,
189            bias=bias,
190            dropout=dropout,
191            direct_link=direct_link,
192            n_clusters=n_clusters,
193            cluster_encode=cluster_encode,
194            type_clust=type_clust,
195            type_scaling=type_scaling,
196            col_sample=col_sample,
197            row_sample=row_sample,
198            seed=seed,
199            backend=backend,
200        )
201
202        self.type_fit = "classification"
203        self.verbose = verbose
204        self.method = method
205        self.reg_lambda = reg_lambda
206        self.reg_alpha = reg_alpha
207
208        self.alpha_ = []
209        self.base_learners_ = dict.fromkeys(range(n_estimators))
210
211    def fit(self, X, y, sample_weight=None, **kwargs):
212        """Fit Boosting model to training data (X, y).
213
214        Parameters:
215
216            X: {array-like}, shape = [n_samples, n_features]
217                Training vectors, where n_samples is the number
218                of samples and n_features is the number of features.
219
220            y: array-like, shape = [n_samples]
221                Target values.
222
223            **kwargs: additional parameters to be passed to
224                    self.cook_training_set or self.obj.fit
225
226        Returns:
227
228             self: object
229        """
230
231        assert mx.is_factor(y), "y must contain only integers"
232
233        assert self.method in (
234            "SAMME",
235            "SAMME.R",
236        ), "`method` must be either 'SAMME' or 'SAMME.R'"
237
238        assert (self.reg_lambda <= 1) & (
239            self.reg_lambda >= 0
240        ), "must have self.reg_lambda <= 1 &  self.reg_lambda >= 0"
241
242        assert (self.reg_alpha <= 1) & (
243            self.reg_alpha >= 0
244        ), "must have self.reg_alpha <= 1 &  self.reg_alpha >= 0"
245
246        # training
247        n, p = X.shape
248        self.n_classes = len(np.unique(y))
249
250        if sample_weight is None:
251            w_m = np.repeat(1.0 / n, n)
252
253        else:
254            w_m = np.asarray(sample_weight)
255
256        base_learner = CustomClassifier(
257            self.obj,
258            n_hidden_features=self.n_hidden_features,
259            activation_name=self.activation_name,
260            a=self.a,
261            nodes_sim=self.nodes_sim,
262            bias=self.bias,
263            dropout=self.dropout,
264            direct_link=self.direct_link,
265            n_clusters=self.n_clusters,
266            type_clust=self.type_clust,
267            type_scaling=self.type_scaling,
268            col_sample=self.col_sample,
269            row_sample=self.row_sample,
270            seed=self.seed,
271        )
272
273        if self.verbose == 1:
274            pbar = Progbar(self.n_estimators)
275
276        if self.method == "SAMME":
277            err_m = 1e6
278            err_bound = 1 - 1 / self.n_classes
279            self.alpha_.append(1.0)
280            x_range_n = range(n)
281
282            for m in range(self.n_estimators):
283                preds = base_learner.fit(
284                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
285                ).predict(X)
286
287                self.base_learners_.update(
288                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
289                )
290
291                cond = [y[i] != preds[i] for i in x_range_n]
292
293                err_m = max(
294                    sum([elt[0] * elt[1] for elt in zip(cond, w_m)]),
295                    2.220446049250313e-16,
296                )  # sum(w_m) == 1
297
298                if self.reg_lambda > 0:
299                    err_m += self.reg_lambda * (
300                        (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m])
301                        + self.reg_alpha * sum([abs(x) for x in w_m])
302                    )
303
304                err_m = min(err_m, err_bound)
305
306                alpha_m = self.learning_rate * log(
307                    (self.n_classes - 1) * (1 - err_m) / err_m
308                )
309
310                self.alpha_.append(alpha_m)
311
312                w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n]
313
314                sum_w_m = sum(w_m_temp)
315
316                w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n])
317
318                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
319
320                if self.verbose == 1:
321                    pbar.update(m)
322
323            if self.verbose == 1:
324                pbar.update(self.n_estimators)
325
326            self.n_estimators = len(self.base_learners_)
327
328            return self
329
330        if self.method == "SAMME.R":
331            Y = mo.one_hot_encode2(y, self.n_classes)
332
333            if sample_weight is None:
334                w_m = np.repeat(1.0 / n, n)  # (N, 1)
335
336            else:
337                w_m = np.asarray(sample_weight)
338
339            for m in range(self.n_estimators):
340                probs = base_learner.fit(
341                    X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs
342                ).predict_proba(X)
343
344                np.clip(
345                    a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs
346                )
347
348                self.base_learners_.update(
349                    {m: pickle.loads(pickle.dumps(base_learner, -1))}
350                )
351
352                w_m *= np.exp(
353                    -1.0
354                    * self.learning_rate
355                    * (1.0 - 1.0 / self.n_classes)
356                    * xlogy(Y, probs).sum(axis=1)
357                )
358
359                w_m /= np.sum(w_m)
360
361                base_learner.set_params(seed=self.seed + (m + 1) * 1000)
362
363                if self.verbose == 1:
364                    pbar.update(m)
365
366            if self.verbose == 1:
367                pbar.update(self.n_estimators)
368
369            self.n_estimators = len(self.base_learners_)
370
371            return self
372
373    def predict(self, X, **kwargs):
374        """Predict test data X.
375
376        Parameters:
377
378            X: {array-like}, shape = [n_samples, n_features]
379                Training vectors, where n_samples is the number
380                of samples and n_features is the number of features.
381
382            **kwargs: additional parameters to be passed to
383                  self.cook_test_set
384
385        Returns:
386
387            model predictions: {array-like}
388        """
389
390        return self.predict_proba(X, **kwargs).argmax(axis=1)
391
392    def predict_proba(self, X, **kwargs):
393        """Predict probabilities for test data X.
394
395        Parameters:
396
397            X: {array-like}, shape = [n_samples, n_features]
398                Training vectors, where n_samples is the number
399                of samples and n_features is the number of features.
400
401            **kwargs: additional parameters to be passed to
402                  self.cook_test_set
403
404        Returns:
405
406            probability estimates for test data: {array-like}
407
408        """
409
410        n_iter = len(self.base_learners_)
411
412        if self.method == "SAMME":
413            ensemble_learner = np.zeros((X.shape[0], self.n_classes))
414
415            # if self.verbose == 1:
416            #    pbar = Progbar(n_iter)
417
418            for idx, base_learner in self.base_learners_.items():
419                preds = base_learner.predict(X, **kwargs)
420
421                ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2(
422                    preds, self.n_classes
423                )
424
425                # if self.verbose == 1:
426                #    pbar.update(idx)
427
428            # if self.verbose == 1:
429            #    pbar.update(n_iter)
430
431            expit_ensemble_learner = expit(ensemble_learner)
432
433            sum_ensemble = expit_ensemble_learner.sum(axis=1)
434
435            return expit_ensemble_learner / sum_ensemble[:, None]
436
437        # if self.method == "SAMME.R":
438        ensemble_learner = 0
439
440        # if self.verbose == 1:
441        #    pbar = Progbar(n_iter)
442
443        for idx, base_learner in self.base_learners_.items():
444            probs = base_learner.predict_proba(X, **kwargs)
445
446            np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs)
447
448            log_preds_proba = np.log(probs)
449
450            ensemble_learner += (
451                log_preds_proba - log_preds_proba.mean(axis=1)[:, None]
452            )
453
454            # if self.verbose == 1:
455            #    pbar.update(idx)
456
457        ensemble_learner *= self.n_classes - 1
458
459        # if self.verbose == 1:
460        #    pbar.update(n_iter)
461
462        expit_ensemble_learner = expit(ensemble_learner)
463
464        sum_ensemble = expit_ensemble_learner.sum(axis=1)
465
466        return expit_ensemble_learner / sum_ensemble[:, None]
467
468    def score(self, X, y, scoring=None, **kwargs):
469        """Score the model on test set features X and response y.
470
471        Parameters:
472
473            X: {array-like}, shape = [n_samples, n_features]
474                Training vectors, where n_samples is the number
475                of samples and n_features is the number of features
476
477            y: array-like, shape = [n_samples]
478                Target values
479
480            scoring: str
481                must be in ('accuracy', 'average_precision',
482                           'brier_score_loss', 'f1', 'f1_micro',
483                           'f1_macro', 'f1_weighted',  'f1_samples',
484                           'neg_log_loss', 'precision', 'recall',
485                           'roc_auc')
486
487            **kwargs: additional parameters to be passed to scoring functions
488
489        Returns:
490
491            model scores: {array-like}
492
493        """
494
495        preds = self.predict(X)
496
497        if scoring is None:
498            scoring = "accuracy"
499
500        # check inputs
501        assert scoring in (
502            "accuracy",
503            "average_precision",
504            "brier_score_loss",
505            "f1",
506            "f1_micro",
507            "f1_macro",
508            "f1_weighted",
509            "f1_samples",
510            "neg_log_loss",
511            "precision",
512            "recall",
513            "roc_auc",
514        ), "'scoring' should be in ('accuracy', 'average_precision', \
515                           'brier_score_loss', 'f1', 'f1_micro', \
516                           'f1_macro', 'f1_weighted',  'f1_samples', \
517                           'neg_log_loss', 'precision', 'recall', \
518                           'roc_auc')"
519
520        scoring_options = {
521            "accuracy": skm2.accuracy_score,
522            "average_precision": skm2.average_precision_score,
523            "brier_score_loss": skm2.brier_score_loss,
524            "f1": skm2.f1_score,
525            "f1_micro": skm2.f1_score,
526            "f1_macro": skm2.f1_score,
527            "f1_weighted": skm2.f1_score,
528            "f1_samples": skm2.f1_score,
529            "neg_log_loss": skm2.log_loss,
530            "precision": skm2.precision_score,
531            "recall": skm2.recall_score,
532            "roc_auc": skm2.roc_auc_score,
533        }
534
535        return scoring_options[scoring](y, preds, **kwargs)

 31class Base(BaseEstimator):
 32    """Base model from which all the other classes inherit.
 33
 34    This class contains the most important data preprocessing/feature engineering methods.
 35
 36    Parameters:
 37
 38        n_hidden_features: int
 39            number of nodes in the hidden layer
 40
 41        activation_name: str
 42            activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
 43
 44        a: float
 45            hyperparameter for 'prelu' or 'elu' activation function
 46
 47        nodes_sim: str
 48            type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
 49            'uniform'
 50
 51        bias: boolean
 52            indicates if the hidden layer contains a bias term (True) or
 53            not (False)
 54
 55        dropout: float
 56            regularization parameter; (random) percentage of nodes dropped out
 57            of the training
 58
 59        direct_link: boolean
 60            indicates if the original features are included (True) in model's
 61            fitting or not (False)
 62
 63        n_clusters: int
 64            number of clusters for type_clust='kmeans' or type_clust='gmm'
 65            clustering (could be 0: no clustering)
 66
 67        cluster_encode: bool
 68            defines how the variable containing clusters is treated (default is one-hot);
 69            if `False`, then labels are used, without one-hot encoding
 70
 71        type_clust: str
 72            type of clustering method: currently k-means ('kmeans') or Gaussian
 73            Mixture Model ('gmm')
 74
 75        type_scaling: a tuple of 3 strings
 76            scaling methods for inputs, hidden layer, and clustering respectively
 77            (and when relevant).
 78            Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust')
 79
 80        col_sample: float
 81            percentage of features randomly chosen for training
 82
 83        row_sample: float
 84            percentage of rows chosen for training, by stratified bootstrapping
 85
 86        seed: int
 87            reproducibility seed for nodes_sim=='uniform', clustering and dropout
 88
 89        backend: str
 90            "cpu" or "gpu" or "tpu"
 91
 92    """
 93
 94    # construct the object -----
 95
 96    def __init__(
 97        self,
 98        n_hidden_features=5,
 99        activation_name="relu",
100        a=0.01,
101        nodes_sim="sobol",
102        bias=True,
103        dropout=0,
104        direct_link=True,
105        n_clusters=2,
106        cluster_encode=True,
107        type_clust="kmeans",
108        type_scaling=("std", "std", "std"),
109        col_sample=1,
110        row_sample=1,
111        seed=123,
112        backend="cpu",
113    ):
114        # input checks -----
115
116        sys_platform = platform.system()
117
118        if (sys_platform == "Windows") and (backend in ("gpu", "tpu")):
119            warnings.warn(
120                "No GPU/TPU computing on Windows yet, backend set to 'cpu'"
121            )
122            backend = "cpu"
123
124        assert activation_name in (
125            "relu",
126            "tanh",
127            "sigmoid",
128            "prelu",
129            "elu",
130        ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')"
131
132        assert nodes_sim in (
133            "sobol",
134            "hammersley",
135            "uniform",
136            "halton",
137        ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')"
138
139        assert type_clust in (
140            "kmeans",
141            "gmm",
142        ), "'type_clust' must be in ('kmeans', 'gmm')"
143
144        assert (len(type_scaling) == 3) & all(
145            type_scaling[i] in ("minmax", "std", "robust")
146            for i in range(len(type_scaling))
147        ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std') and robust scaling ('robust')"
148
149        assert (col_sample >= 0) & (
150            col_sample <= 1
151        ), "'col_sample' must be comprised between 0 and 1 (both included)"
152
153        assert backend in (
154            "cpu",
155            "gpu",
156            "tpu",
157        ), "must have 'backend' in ('cpu', 'gpu', 'tpu')"
158
159        self.n_hidden_features = n_hidden_features
160        self.activation_name = activation_name
161        self.a = a
162        self.nodes_sim = nodes_sim
163        self.bias = bias
164        self.seed = seed
165        self.backend = backend
166        self.dropout = dropout
167        self.direct_link = direct_link
168        self.cluster_encode = cluster_encode
169        self.type_clust = type_clust
170        self.type_scaling = type_scaling
171        self.col_sample = col_sample
172        self.row_sample = row_sample
173        self.n_clusters = n_clusters
174        self.subsampler_ = None
175        self.index_col_ = None
176        self.index_row_ = True
177        self.clustering_obj_ = None
178        self.clustering_scaler_ = None
179        self.nn_scaler_ = None
180        self.scaler_ = None
181        self.encoder_ = None
182        self.W_ = None
183        self.X_ = None
184        self.y_ = None
185        self.y_mean_ = None
186        self.beta_ = None
187
188        # activation function -----
189        if sys_platform in ("Linux", "Darwin"):
190            activation_options = {
191                "relu": ac.relu if (self.backend == "cpu") else jnn.relu,
192                "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh,
193                "sigmoid": (
194                    ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid
195                ),
196                "prelu": partial(ac.prelu, a=a),
197                "elu": (
198                    partial(ac.elu, a=a)
199                    if (self.backend == "cpu")
200                    else partial(jnn.elu, a=a)
201                ),
202            }
203        else:  # on Windows currently, no JAX
204            activation_options = {
205                "relu": (
206                    ac.relu if (self.backend == "cpu") else NotImplementedError
207                ),
208                "tanh": (
209                    np.tanh if (self.backend == "cpu") else NotImplementedError
210                ),
211                "sigmoid": (
212                    ac.sigmoid
213                    if (self.backend == "cpu")
214                    else NotImplementedError
215                ),
216                "prelu": partial(ac.prelu, a=a),
217                "elu": (
218                    partial(ac.elu, a=a)
219                    if (self.backend == "cpu")
220                    else NotImplementedError
221                ),
222            }
223        self.activation_func = activation_options[activation_name]
224
225    # "preprocessing" methods to be inherited -----
226
227    def encode_clusters(self, X=None, predict=False, **kwargs):  #
228        """Create new covariates with kmeans or GMM clustering
229
230        Parameters:
231
232            X: {array-like}, shape = [n_samples, n_features]
233                Training vectors, where n_samples is the number
234                of samples and n_features is the number of features.
235
236            predict: boolean
237                is False on training set and True on test set
238
239            **kwargs:
240                additional parameters to be passed to the
241                clustering method
242
243        Returns:
244
245            Clusters' matrix, one-hot encoded: {array-like}
246
247        """
248
249        np.random.seed(self.seed)
250
251        if X is None:
252            X = self.X_
253
254        if isinstance(X, pd.DataFrame):
255            X = copy.deepcopy(X.values.astype(float))
256
257        if predict is False:  # encode training set
258            # scale input data before clustering
259            self.clustering_scaler_, scaled_X = mo.scale_covariates(
260                X, choice=self.type_scaling[2]
261            )
262
263            self.clustering_obj_, X_clustered = mo.cluster_covariates(
264                scaled_X,
265                self.n_clusters,
266                self.seed,
267                type_clust=self.type_clust,
268                **kwargs
269            )
270
271            if self.cluster_encode == True:
272                return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
273                    np.float16
274                )
275
276            return X_clustered.astype(np.float16)
277
278        # if predict == True, encode test set
279        X_clustered = self.clustering_obj_.predict(
280            self.clustering_scaler_.transform(X)
281        )
282
283        if self.cluster_encode == True:
284            return mo.one_hot_encode(X_clustered, self.n_clusters).astype(
285                np.float16
286            )
287
288        return X_clustered.astype(np.float16)
289
290    def create_layer(self, scaled_X, W=None):
291        """Create hidden layer.
292
293        Parameters:
294
295            scaled_X: {array-like}, shape = [n_samples, n_features]
296                Training vectors, where n_samples is the number
297                of samples and n_features is the number of features
298
299            W: {array-like}, shape = [n_features, hidden_features]
300                if provided, constructs the hidden layer with W; otherwise computed internally
301
302        Returns:
303
304            Hidden layer matrix: {array-like}
305
306        """
307
308        n_features = scaled_X.shape[1]
309
310        # hash_sim = {
311        #         "sobol": generate_sobol,
312        #         "hammersley": generate_hammersley,
313        #         "uniform": generate_uniform,
314        #         "halton": generate_halton
315        #     }
316
317        if self.bias is False:  # no bias term in the hidden layer
318            if W is None:
319                if self.nodes_sim == "sobol":
320                    self.W_ = generate_sobol(
321                        n_dims=n_features,
322                        n_points=self.n_hidden_features,
323                        seed=self.seed,
324                    )
325                elif self.nodes_sim == "hammersley":
326                    self.W_ = generate_hammersley(
327                        n_dims=n_features,
328                        n_points=self.n_hidden_features,
329                        seed=self.seed,
330                    )
331                elif self.nodes_sim == "uniform":
332                    self.W_ = generate_uniform(
333                        n_dims=n_features,
334                        n_points=self.n_hidden_features,
335                        seed=self.seed,
336                    )
337                else:
338                    self.W_ = generate_halton(
339                        n_dims=n_features,
340                        n_points=self.n_hidden_features,
341                        seed=self.seed,
342                    )
343
344                # self.W_ = hash_sim[self.nodes_sim](
345                #             n_dims=n_features,
346                #             n_points=self.n_hidden_features,
347                #             seed=self.seed,
348                #         )
349
350                assert (
351                    scaled_X.shape[1] == self.W_.shape[0]
352                ), "check dimensions of covariates X and matrix W"
353
354                return mo.dropout(
355                    x=self.activation_func(
356                        mo.safe_sparse_dot(
357                            a=scaled_X, b=self.W_, backend=self.backend
358                        )
359                    ),
360                    drop_prob=self.dropout,
361                    seed=self.seed,
362                )
363
364            # W is not none
365            assert (
366                scaled_X.shape[1] == W.shape[0]
367            ), "check dimensions of covariates X and matrix W"
368
369            # self.W_ = W
370            return mo.dropout(
371                x=self.activation_func(
372                    mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend)
373                ),
374                drop_prob=self.dropout,
375                seed=self.seed,
376            )
377
378        # with bias term in the hidden layer
379        if W is None:
380            n_features_1 = n_features + 1
381
382            if self.nodes_sim == "sobol":
383                self.W_ = generate_sobol(
384                    n_dims=n_features_1,
385                    n_points=self.n_hidden_features,
386                    seed=self.seed,
387                )
388            elif self.nodes_sim == "hammersley":
389                self.W_ = generate_hammersley(
390                    n_dims=n_features_1,
391                    n_points=self.n_hidden_features,
392                    seed=self.seed,
393                )
394            elif self.nodes_sim == "uniform":
395                self.W_ = generate_uniform(
396                    n_dims=n_features_1,
397                    n_points=self.n_hidden_features,
398                    seed=self.seed,
399                )
400            else:
401                self.W_ = generate_halton(
402                    n_dims=n_features_1,
403                    n_points=self.n_hidden_features,
404                    seed=self.seed,
405                )
406
407            # self.W_ = hash_sim[self.nodes_sim](
408            #         n_dims=n_features_1,
409            #         n_points=self.n_hidden_features,
410            #         seed=self.seed,
411            #     )
412
413            return mo.dropout(
414                x=self.activation_func(
415                    mo.safe_sparse_dot(
416                        a=mo.cbind(
417                            np.ones(scaled_X.shape[0]),
418                            scaled_X,
419                            backend=self.backend,
420                        ),
421                        b=self.W_,
422                        backend=self.backend,
423                    )
424                ),
425                drop_prob=self.dropout,
426                seed=self.seed,
427            )
428
429        # W is not None
430        # self.W_ = W
431        return mo.dropout(
432            x=self.activation_func(
433                mo.safe_sparse_dot(
434                    a=mo.cbind(
435                        np.ones(scaled_X.shape[0]),
436                        scaled_X,
437                        backend=self.backend,
438                    ),
439                    b=W,
440                    backend=self.backend,
441                )
442            ),
443            drop_prob=self.dropout,
444            seed=self.seed,
445        )
446
447    def cook_training_set(self, y=None, X=None, W=None, **kwargs):
448        """Create new hidden features for training set, with hidden layer, center the response.
449
450        Parameters:
451
452            y: array-like, shape = [n_samples]
453                Target values
454
455            X: {array-like}, shape = [n_samples, n_features]
456                Training vectors, where n_samples is the number
457                of samples and n_features is the number of features
458
459            W: {array-like}, shape = [n_features, hidden_features]
460                if provided, constructs the hidden layer via W
461
462        Returns:
463
464            (centered response, direct link + hidden layer matrix): {tuple}
465
466        """
467
468        # either X and y are stored or not
469        # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None))
470        if self.n_hidden_features > 0:  # has a hidden layer
471            assert (
472                len(self.type_scaling) >= 2
473            ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0"
474
475        if X is None:
476            if self.col_sample == 1:
477                input_X = self.X_
478            else:
479                n_features = self.X_.shape[1]
480                new_n_features = int(np.ceil(n_features * self.col_sample))
481                assert (
482                    new_n_features >= 1
483                ), "check class attribute 'col_sample' and the number of covariates provided for X"
484                np.random.seed(self.seed)
485                index_col = np.random.choice(
486                    range(n_features), size=new_n_features, replace=False
487                )
488                self.index_col_ = index_col
489                input_X = self.X_[:, self.index_col_]
490
491        else:  # X is not None # keep X vs self.X_
492            if isinstance(X, pd.DataFrame):
493                X = copy.deepcopy(X.values.astype(float))
494
495            if self.col_sample == 1:
496                input_X = X
497            else:
498                n_features = X.shape[1]
499                new_n_features = int(np.ceil(n_features * self.col_sample))
500                assert (
501                    new_n_features >= 1
502                ), "check class attribute 'col_sample' and the number of covariates provided for X"
503                np.random.seed(self.seed)
504                index_col = np.random.choice(
505                    range(n_features), size=new_n_features, replace=False
506                )
507                self.index_col_ = index_col
508                input_X = X[:, self.index_col_]
509
510        if (
511            self.n_clusters <= 0
512        ):  # data without any clustering: self.n_clusters is None -----
513            if self.n_hidden_features > 0:  # with hidden layer
514                self.nn_scaler_, scaled_X = mo.scale_covariates(
515                    input_X, choice=self.type_scaling[1]
516                )
517                Phi_X = (
518                    self.create_layer(scaled_X)
519                    if W is None
520                    else self.create_layer(scaled_X, W=W)
521                )
522                Z = (
523                    mo.cbind(input_X, Phi_X, backend=self.backend)
524                    if self.direct_link is True
525                    else Phi_X
526                )
527                self.scaler_, scaled_Z = mo.scale_covariates(
528                    Z, choice=self.type_scaling[0]
529                )
530            else:  # no hidden layer
531                Z = input_X
532                self.scaler_, scaled_Z = mo.scale_covariates(
533                    Z, choice=self.type_scaling[0]
534                )
535        else:  # data with clustering: self.n_clusters is not None ----- # keep
536            augmented_X = mo.cbind(
537                input_X,
538                self.encode_clusters(input_X, **kwargs),
539                backend=self.backend,
540            )
541
542            if self.n_hidden_features > 0:  # with hidden layer
543                self.nn_scaler_, scaled_X = mo.scale_covariates(
544                    augmented_X, choice=self.type_scaling[1]
545                )
546                Phi_X = (
547                    self.create_layer(scaled_X)
548                    if W is None
549                    else self.create_layer(scaled_X, W=W)
550                )
551                Z = (
552                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
553                    if self.direct_link is True
554                    else Phi_X
555                )
556                self.scaler_, scaled_Z = mo.scale_covariates(
557                    Z, choice=self.type_scaling[0]
558                )
559            else:  # no hidden layer
560                Z = augmented_X
561                self.scaler_, scaled_Z = mo.scale_covariates(
562                    Z, choice=self.type_scaling[0]
563                )
564
565        # Returning model inputs -----
566        if mx.is_factor(y) is False:  # regression
567            # center y
568            if y is None:
569                self.y_mean_, centered_y = mo.center_response(self.y_)
570            else:
571                self.y_mean_, centered_y = mo.center_response(y)
572
573            # y is subsampled
574            if self.row_sample < 1:
575                n, p = Z.shape
576
577                self.subsampler_ = (
578                    SubSampler(
579                        y=self.y_, row_sample=self.row_sample, seed=self.seed
580                    )
581                    if y is None
582                    else SubSampler(
583                        y=y, row_sample=self.row_sample, seed=self.seed
584                    )
585                )
586
587                self.index_row_ = self.subsampler_.subsample()
588
589                n_row_sample = len(self.index_row_)
590                # regression
591                return (
592                    centered_y[self.index_row_].reshape(n_row_sample),
593                    self.scaler_.transform(
594                        Z[self.index_row_, :].reshape(n_row_sample, p)
595                    ),
596                )
597            # y is not subsampled
598            # regression
599            return (centered_y, self.scaler_.transform(Z))
600
601        # classification
602        # y is subsampled
603        if self.row_sample < 1:
604            n, p = Z.shape
605
606            self.subsampler_ = (
607                SubSampler(
608                    y=self.y_, row_sample=self.row_sample, seed=self.seed
609                )
610                if y is None
611                else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed)
612            )
613
614            self.index_row_ = self.subsampler_.subsample()
615
616            n_row_sample = len(self.index_row_)
617            # classification
618            return (
619                y[self.index_row_].reshape(n_row_sample),
620                self.scaler_.transform(
621                    Z[self.index_row_, :].reshape(n_row_sample, p)
622                ),
623            )
624        # y is not subsampled
625        # classification
626        return (y, self.scaler_.transform(Z))
627
628    def cook_test_set(self, X, **kwargs):
629        """Transform data from test set, with hidden layer.
630
631        Parameters:
632
633            X: {array-like}, shape = [n_samples, n_features]
634                Training vectors, where n_samples is the number
635                of samples and n_features is the number of features
636
637            **kwargs: additional parameters to be passed to self.encode_cluster
638
639        Returns:
640
641            Transformed test set : {array-like}
642        """
643
644        if isinstance(X, pd.DataFrame):
645            X = copy.deepcopy(X.values.astype(float))
646
647        if (
648            self.n_clusters == 0
649        ):  # data without clustering: self.n_clusters is None -----
650            if self.n_hidden_features > 0:
651                # if hidden layer
652                scaled_X = (
653                    self.nn_scaler_.transform(X)
654                    if (self.col_sample == 1)
655                    else self.nn_scaler_.transform(X[:, self.index_col_])
656                )
657                Phi_X = self.create_layer(scaled_X, self.W_)
658                if self.direct_link == True:
659                    return self.scaler_.transform(
660                        mo.cbind(scaled_X, Phi_X, backend=self.backend)
661                    )
662                # when self.direct_link == False
663                return self.scaler_.transform(Phi_X)
664            # if no hidden layer # self.n_hidden_features == 0
665            return self.scaler_.transform(X)
666
667        # data with clustering: self.n_clusters > 0 -----
668        if self.col_sample == 1:
669            predicted_clusters = self.encode_clusters(
670                X=X, predict=True, **kwargs
671            )
672            augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend)
673        else:
674            predicted_clusters = self.encode_clusters(
675                X=X[:, self.index_col_], predict=True, **kwargs
676            )
677            augmented_X = mo.cbind(
678                X[:, self.index_col_], predicted_clusters, backend=self.backend
679            )
680
681        if self.n_hidden_features > 0:  # if hidden layer
682            scaled_X = self.nn_scaler_.transform(augmented_X)
683            Phi_X = self.create_layer(scaled_X, self.W_)
684            if self.direct_link == True:
685                return self.scaler_.transform(
686                    mo.cbind(augmented_X, Phi_X, backend=self.backend)
687                )
688            return self.scaler_.transform(Phi_X)
689
690        # if no hidden layer
691        return self.scaler_.transform(augmented_X)